# encoding=utf-8
# ---------------------------------------
#   版本：0.1
#   日期：2016-11-10
#   作者：九茶<bone_ace@163.com>
# ---------------------------------------


BOT_NAME = 'scrapyWebSite'
SPIDER_MODULES = ['scrapyWebSite.spiders']
NEWSPIDER_MODULE = 'scrapyWebSite.spiders'

SCHEDULER = 'scrapyWebSite.scrapy_redis.scheduler.Scheduler'
SCHEDULER_PERSIST = True
SCHEDULER_QUEUE_CLASS = 'scrapyWebSite.scrapy_redis.queue.SpiderPriorityQueue'
# SCHEDULER_QUEUE_CLASS = 'scrapyWithBloomfilter_demo.scrapy_redis.queue.SpiderSimpleQueue'

# 种子队列的信息
REDIE_URL = None
REDIS_HOST = 'localhost'
REDIS_PORT = 6379

# 去重队列的信息
FILTER_URL = None
FILTER_HOST = 'localhost'
FILTER_PORT = 6379
FILTER_DB = 0
# REDIS_QUEUE_NAME = 'OneName'   # 如果不设置或者设置为None，则使用默认的，每个spider使用不同的去重队列和种子队列。如果设置了，则不同spider共用去重队列和种子队列

MYSQL_HOST = 'localhost'
MYSQL_PORT = 3306
MYSQL_DBNAME = "webmgr_data"
MYSQL_USER = 'root'
MYSQL_PASSWORD = '123456'

LOG_LEVEL = 'INFO'
CONCURRENT_REQUESTS = 4  # 并发

ITEM_PIPELINES = {
    'scrapyWebSite.pipelines.MysqlPipeline': 300,
}
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) ' \
             'Chrome/55.0.2883.95 Safari/537.36 '
CONCURRENT_REQUESTS = 1
DOWNLOAD_DELAY = 2.5
MYEXT_ENABLED = True  # 开启扩展
IDLE_NUMBER = 6  # 配置空闲持续时间单位为 360个 ，一个时间单位为5s

# 在 EXTENSIONS 配置，激活扩展
EXTENSIONS = {
    'scrapyWebSite.extensions.RedisSpiderSmartIdleClosedExensions': 500,
}
